from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np

'''
percentageList = [0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.1,0.15,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0]
# building the log cost model for UP / TCP
termAndUnprunedListLengthDict = {}
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/partOfLexiconTermsWithTermIDForTail5KQueries"
ifh = open(ifn,"r")
for line in ifh.readlines():
    lineElements = line.strip().split(" ")
    currTerm = lineElements[1]
    currTermUnprunedListLength = int(lineElements[2])
    termAndUnprunedListLengthDict[currTerm] = currTermUnprunedListLength
print "len(termAndUnprunedListLengthDict):",len(termAndUnprunedListLengthDict)
ifh.close()

# key: qid
# value: a list full of query terms
qidAndQueryTermsDict = {}
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/GOV2_100KQueries_tail_5K_formattedForLogQueryCostModel"
ifh = open(ifn,"r")
for line in ifh.readlines():
    qid = line.strip().split(":")[0]
    queryContent = line.strip().split(":")[1]
    queryTerms = queryContent.split(" ")
    qidAndQueryTermsDict[qid] = queryTerms 
print "len(qidAndQueryTermsDict):",len(qidAndQueryTermsDict)
ifh.close()

for index,currPerValue in enumerate(percentageList):
    totalQPCCostLogModel = 0
    totalQPCCostSumModel = 0
    for qid in qidAndQueryTermsDict:
        listLengthList = []
        for term in qidAndQueryTermsDict[qid]:
            if term in termAndUnprunedListLengthDict:
                listLengthList.append((term,termAndUnprunedListLengthDict[term]))
        listLengthList.sort(cmp=None, key=itemgetter(1), reverse=False)
        if len(listLengthList) >= 2:
            (_,l0) = listLengthList[0]
            (_,l1) = listLengthList[1]
            l0 = l0 * currPerValue
            l1 = l1 * currPerValue
            currentQPCCostLogModel = l0 * math.log(l1 / l0 + 1)
            # print qid,listLengthList,currentQPCCostLogModel
            totalQPCCostLogModel += currentQPCCostLogModel
        currentQPCCostSumModel = 0
        for tuple in listLengthList:
            (_,l) = tuple
            currentQPCCostSumModel += l * currPerValue
        totalQPCCostSumModel += currentQPCCostSumModel
    # print "len(termAndUnprunedListLengthDict):",len(termAndUnprunedListLengthDict)
    print "----->",percentageList[index],totalQPCCostSumModel,totalQPCCostLogModel
exit(1)
'''

# The below are building the cost model for our approach
print "Begins..."
termAndUnprunedListLengthDict = {}
ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/partOfLexiconTermsWithTermIDForTail5KQueries"
ifh = open(ifn,"r")
for line in ifh.readlines():
    lineElements = line.strip().split(" ")
    currTerm = lineElements[1]
    currTermUnprunedListLength = int(lineElements[2])
    termAndUnprunedListLengthDict[currTerm] = currTermUnprunedListLength
print "len(termAndUnprunedListLengthDict):",len(termAndUnprunedListLengthDict) 
ifh.close()


nameList = ["ptPowTo0","ptPowToDot1","ptPowToDot3","ptPowToDot5","ptPowToDot7","ptPowToDot9","ptPowTo1"]
for currentPowValue in nameList:
        '''
        # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_0"
        ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_1"
        ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_2"
        ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_3"
        ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_4"
        ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_5"
        ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_6"
        ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_7"
        ifn8 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_8"
        ifn9 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_9"
        ifn10 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_10"
        ifn11 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_11"
        ifn12 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_12"
        ifn13 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_13"
        ifn14 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_14"
        ifn15 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_15"
        ifn16 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_16"
        ifn17 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_17"
        ifn18 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_18"
        ifn19 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_0_" + currentPowValue + "_termAndPrunedListLength_20140720/queryTerms_dynamicWeight_0_" + currentPowValue + "_final_19"
        '''
        
        # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_0"
        ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_1"
        ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_2"
        ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_3"
        ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_4"
        ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_5"
        ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_6"
        ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_7"
        ifn8 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_8"
        ifn9 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_9"
        ifn10 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_10"
        ifn11 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_11"
        ifn12 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_12"
        ifn13 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_13"
        ifn14 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_14"
        ifn15 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_15"
        ifn16 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_16"
        ifn17 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_17"
        ifn18 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_18"
        ifn19 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/dynamicWeight_5_" + currentPowValue + "_termAndPrunedListLength_20140723/queryTerms_dynamicWeight_5_" + currentPowValue + "_final_19"
        
        ifnNameList = []
        # ifnNameList.append(ifn0)
        ifnNameList.append(ifn1)
        ifnNameList.append(ifn2)
        ifnNameList.append(ifn3)
        ifnNameList.append(ifn4)
        ifnNameList.append(ifn5)
        ifnNameList.append(ifn6)
        ifnNameList.append(ifn7)
        ifnNameList.append(ifn8)
        ifnNameList.append(ifn9)
        ifnNameList.append(ifn10)
        ifnNameList.append(ifn11)
        ifnNameList.append(ifn12)
        ifnNameList.append(ifn13)
        ifnNameList.append(ifn14)
        ifnNameList.append(ifn15)
        ifnNameList.append(ifn16)
        ifnNameList.append(ifn17)
        ifnNameList.append(ifn18)
        ifnNameList.append(ifn19)


        # key: qid
        # value: a list full of query terms
        qidAndQueryTermsDict = {}
        ifn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/GOV2_100KQueries_tail_5K_formattedForLogQueryCostModel"
        ifh = open(ifn,"r")
        for line in ifh.readlines():
            qid = line.strip().split(":")[0]
            queryContent = line.strip().split(":")[1]
            queryTerms = queryContent.split(" ")
            qidAndQueryTermsDict[qid] = queryTerms 
        print "len(qidAndQueryTermsDict):",len(qidAndQueryTermsDict)
        ifh.close()

        totalQPCCostLogModel = 0
        totalQPCCostSumModel = 0
        percentageList = ["1%","2%","3%","4%","5%","6%","7%","8%","9%","10%","15%","20%","30%","40%","50%","60%","70%","80%","90%"]
        for index,currentInputName in enumerate(ifnNameList):
            print "currentInputName:",currentInputName
            totalQPCCostLogModel = 0
            totalQPCCostSumModel = 0
            termAndPrunedListLengthDict = {}
            
            ifh = open(currentInputName,"r")
            currentLine = ifh.readline()
            while currentLine:
                currentLineElements = currentLine.strip().split(" ")
                currentTerm = currentLineElements[2]
                currentTermPrunedListLength = int(currentLineElements[1])
                currentTermUnprunedListLength = int(currentLineElements[3])
                termAndPrunedListLengthDict[currentTerm] = currentTermPrunedListLength
                
                currentLine = ifh.readline()
            print "len(termAndPrunedListLengthDict):",len(termAndPrunedListLengthDict)
            
            # start to compute the query processing currentQPCCostLogModel for each query
            for qid in qidAndQueryTermsDict:
                listLengthList = []
                for term in qidAndQueryTermsDict[qid]:
                    if term in termAndPrunedListLengthDict:
                        listLengthList.append((term,termAndPrunedListLengthDict[term]))
                listLengthList.sort(cmp=None, key=itemgetter(1), reverse=False)
                if len(listLengthList) >= 2:
                    (_,l0) = listLengthList[0]
                    (_,l1) = listLengthList[1]
                    currentQPCCostLogModel = l0 * math.log(l1 / l0)
                    # print qid,listLengthList,currentQPCCostLogModel
                    totalQPCCostLogModel += currentQPCCostLogModel
                
                currentQPCCostSumModel = 0
                for tuple in listLengthList:
                    (_,l) = tuple
                    currentQPCCostSumModel += l
                totalQPCCostSumModel += currentQPCCostSumModel
            print "----->",currentPowValue,percentageList[index],totalQPCCostSumModel,totalQPCCostLogModel
            print

        totalQPCCostLogModel = 0
        totalQPCCostSumModel = 0
        for qid in qidAndQueryTermsDict:
            listLengthList = []
            for term in qidAndQueryTermsDict[qid]:
                if term in termAndUnprunedListLengthDict:
                    listLengthList.append((term,termAndUnprunedListLengthDict[term]))
            listLengthList.sort(cmp=None, key=itemgetter(1), reverse=False)
            if len(listLengthList) >= 2:
                (_,l0) = listLengthList[0]
                (_,l1) = listLengthList[1]
                currentQPCCostLogModel = l0 * math.log(l1 / l0 + 1)
                # print qid,listLengthList,currentQPCCostLogModel
                totalQPCCostLogModel += currentQPCCostLogModel
            currentQPCCostSumModel = 0
            for tuple in listLengthList:
                (_,l) = tuple
                currentQPCCostSumModel += l
            totalQPCCostSumModel += currentQPCCostSumModel
        print "len(termAndUnprunedListLengthDict):",len(termAndUnprunedListLengthDict)
        print "----->",currentPowValue,"100%",totalQPCCostSumModel,totalQPCCostLogModel
        print "Ends."










